import os
import re
import string
from collections import Counter

import matplotlib.pyplot as plt
import nltk.data
import plotly.graph_objs as go

import plotly.offline as py
import numpy as np
import pandas as pd

import seaborn as sns
# from bokeh.io import show
# from bokeh.models import HoverTool, ColumnDataSource
from nltk import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from sklearn.cluster import MiniBatchKMeans
from sklearn.decomposition import TruncatedSVD, LatentDirichletAllocation
from sklearn.manifold import TSNE
from spacy.lang.en import stop_words
# from wordcloud import WordCloud
# import bokeh.plotting as bp

sns.set(style="white")

import warnings

warnings.filterwarnings('ignore')
import logging
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

logging.getLogger('lda').setLevel(logging.WARNING)
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
# 设置pandas显示配置
pd.set_option('display.max_columns', 1000)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)

train = pd.read_csv('./train.tsv', sep='\t')
test = pd.read_csv('./test.tsv', sep='\t')
# # 查看数据集大小
# print(train.shape)
# print(test.shape)
# # 查看数据集列名
# print(train.dtypes)
# # 查看数据集前几行
# print(train.head())
# # 查看商品价格信息
# print(train.price.describe())
#
# # 对价格进行对数变换，比较转换前和转换后的分布情况
# plt.subplot(1, 2, 1)
# (train['price']).plot.hist(bins=50, figsize=(20, 10), edgecolor='white', range=[0, 250])
# plt.xlabel('price+', fontsize=17)
# plt.ylabel('frequency', fontsize=17)
# plt.tick_params(labelsize=15)
# plt.title('Price Distribution - Training Set', fontsize=17)
#
# plt.subplot(1, 2, 2)
# np.log(train['price'] + 1).plot.hist(bins=50, figsize=(20, 10), edgecolor='white')
# plt.xlabel('log(price+1)', fontsize=17)
# plt.ylabel('frequency', fontsize=17)
# plt.tick_params(labelsize=15)
# plt.title('Log(Price) Distribution - Training Set', fontsize=17)
# plt.show()
# # 运费承担：大概有55%卖家承担运费
# print(train.shipping.value_counts() / len(train))
# # 看下运费不同情况下的价格变化(包邮的价格贵一些）
# prc_shipBySeller = train.loc[train.shipping == 0, 'price']
# prc_shipByBuyer = train.loc[train.shipping == 1, 'price']
# fig, ax = plt.subplots(figsize=(20, 10))
# ax.hist(np.log(prc_shipBySeller + 1), color='#8CB4E1', alpha=1.0, bins=50, label='Price when Seller pays Shipping')
# ax.hist(np.log(prc_shipByBuyer + 1), color='#007D00', alpha=0.7, bins=50, label='Price when Buyer pays Shipping')
# ax.set(title='Histogram Comparison', ylabel='% of Dataset in Bin')
# plt.legend()
# plt.xlabel('log(price+1)', fontsize=17)
# plt.ylabel('frequency', fontsize=17)
# plt.title('Price Distribution by Shipping Type', fontsize=17)
# plt.tick_params(labelsize=15)
# plt.show()
# 商品类别划分
# print('There are %d unique values in the category column' % train['category_name'].nunique())
# print(train['category_name'].value_counts()[:5])
# print('There are %d items that do not have a label' % train['category_name'].isnull().sum())


# 商品类别太多了，合并一下
def split_cat(text):
    try:
        return text.split("/")
    except:
        return "No Label", "No Label", "No Label"

train['general_cat'], train['subcat_1'], train['subcat_2'] = zip(*train['category_name'].apply(lambda x: split_cat(x)))
# print(train.head())
test['general_cat'], test['subcat_1'], test['subcat_2'] = zip(*test['category_name'].apply(lambda x: split_cat(x)))
# print('There are %d unique general_cat' % train['general_cat'].nunique())
# print('There are %d unique first sub-categories' % train['subcat_1'].nunique())
# print('There are %d unique second sub-categories' % train['subcat_2'].nunique())
# 主类别分布情况
# x = train['general_cat'].value_counts().index.values.astype('str')
# y = train['general_cat'].value_counts().values
# pct = [('%.2f' % (v * 100)) + '%' for v in (y / len(train))]
# tracel = go.Bar(x=x, y=y, text=pct)
# layout = dict(title="Number of Items by Main Category", yaxis=dict(title='Count'), xaxis=dict(title='Category'))
# fig = dict(data=[tracel], layout=layout)
# py.plot(fig)
# # 前15个子类别分布情况
# x = train['subcat_1'].value_counts().index.values.astype('str')[:15]
# y = train['subcat_1'].value_counts().values[:15]
# pct = [('%.2f' % (v * 100)) + '%' for v in (y / len(train))][:15]
# tracel = go.Bar(x=x, y=y, text=pct, marker=dict(color=y, colorscale='Portland', showscale=True, reversescale=False))
# layout = dict(title="Number of Items by Sub Category(Top 15)", yaxis=dict(title='Count'),
#               xaxis=dict(title='SubCategory'))
# fig = dict(data=[tracel], layout=layout)
# py.plot(fig)
# 不同类型商品价格浮动区间
general_cats = train['general_cat'].unique()
x = [train.loc[train['general_cat'] == cat, 'price'] for cat in general_cats]
# data = [go.Box(x=np.log(x[i] + 1), name=general_cats[i]) for i in range(len(general_cats))]
# layout = dict(title='Price Distribution by General Category', yaxis=dict(title='Frequency'),
#               xaxis=dict(title='Category'))
# fig = dict(data=data, layout=layout)
# py.plot(fig)#py.iplot(fig)
# # 前10品牌名称的数据分布
# x = train['brand_name'].value_counts().index.values.astype('str')[:10]
# y = train['brand_name'].value_counts().values[:10]
# tracel = go.Bar(x=x, y=y, marker=dict(color=y, colorscale='Portland', showscale=True, reversescale=False))
# layout = dict(title="Top 10 Brand by Number of Items", yaxis=dict(title='Count'), xaxis=dict(title='Brand Name'))
# fig = dict(data=[tracel], layout=layout)
# py.plot(fig)


# 商品描述对价格的影响
def wordCount(text):
    try:
        text = text.lower()
        regex = re.compile('[' + re.escape(string.punctuation) + '0-9\\r\\t\\n]')
        txt = regex.sub(' ', text)
        words = [w for w in txt.split(" ") if w not in stop_words.STOP_WORDS and len(w) > 3]
        return len(words)
    except:
        return 0


train['desc_len'] = train['item_description'].apply(lambda x: wordCount(x))
test['desc_len'] = test['item_description'].apply(lambda x: wordCount(x))
# print(train.head())

df = train.groupby('desc_len')['price'].mean().reset_index()
# tracel = go.Scatter(x=df['desc_len'], y=np.log(df['price'] + 1), mode='lines+markers', name='lines+markers')
# layout = dict(title='Average Log(Price) by Description Length', yaxis=dict(title='Average Log(Price)'),
#               xaxis=dict(title='Description Length'))
# fig = dict(data=[tracel], layout=layout)
# py.iplot(fig)
# print(train.item_description.isnull().sum())
# 去掉缺失值
train = train[pd.notnull(train['item_description'])]
general_cats = train['general_cat'].unique()
x = [train.loc[train['general_cat'] == cat, 'price'] for cat in general_cats]
# 提取每种品牌的描述关键词
tokenize = nltk.data.load('tokenizers/punkt/english.pickle')
cat_desc = dict()
for cat in general_cats:
    text = ' '.join(train.loc[train['general_cat'] == cat, 'item_description'].values)
    cat_desc[cat] = tokenize.tokenize(text)

#
# 统计常用关键词
flat_lst = [item for sublist in list(cat_desc.values()) for item in sublist]

allWordsCount = Counter(flat_lst)
all_top10 = allWordsCount.most_common(20)
x = [w[0] for w in all_top10]
y = [w[1] for w in all_top10]
tracel = go.Bar(x=x, y=y)
layout = dict(title='Word Frequency', yaxis=dict(title='Count'), xaxis=dict(title='Word'))
fig = dict(data=[tracel], layout=layout)
py.iplot(fig)